#include "DictProducer.h"
#include "Mylogger.h"
#include "SplitTool.h"
#include <stdio.h>
#include <stdlib.h>
#include <iostream>
#include <fstream>
#include <sstream>
#include <unordered_map>

using std::ifstream;
using std::istream;
using std::unordered_map;
using std::ofstream;

//------------------------------预处理-------------------------------
size_t DictProducer::nBytesCode(const char ch){
    if(ch & (1<<7)){
        int nBytes = 1;
        for(int idx = 0;idx != 6; ++idx){
            if(ch & (1<<(6-idx))){
                ++nBytes;
            }else{
                break;
            }
        }
        return nBytes;
    }
    return 1;
}

//创建停用词的set
void DictProducer::buildStopSet(){
    /* std::cout << "buildStopSet :" <<std::endl; */
    for(string filename:_conf._stopFiles){
        //打开文件
        ifstream ifs(filename);
        if(!ifs.good()){
            std::cout << "open " << filename << " fail "<<std::endl;
            return ;
        }
        //读出一个单词
        string word;
        while(ifs >> word){
            //加入set
            _stopWords.insert(word);
        }
        //关闭文件
        ifs.close();
    }
}

//-------------------------------------------------------------------
void DictProducer::buildEnDict(){
    //用来保存单词及其出现的次数
    /* std::cout << "buildEnDict :" <<std::endl; */
    unordered_map<string,int> wordfreq;
    for(string filename:_conf._engFiles){
        //1.打开文件
        /* std::cout << "filename :" << filename <<std::endl; */
        ifstream ifs(filename);
        if(!ifs.good()){
            std::cout << "open " << filename << " fail "<<std::endl;
            return ;
        }
        //2.从文件中读取出一行
        string line;
        while(getline(ifs,line)){
            //3.对数据进行清洗(大写-->小写,标点-->空格,'\r'-->空格)
            for(int idx =0;idx <(int)line.size();++idx){
                if(line[idx]=='\r'||ispunct(line[idx])){
                    line[idx] = ' ';
                }else if(isupper(line[idx])){
                    line[idx] = tolower(line[idx]);
                }
            }
            //4.对清洗完的数据进行切割,将单词和词频加入unordered_map中
            std::istringstream iss(line);
            string word;
            while(iss >> word){
                //如果单词没有出现在停止中，就加入unordered_map
                int count = _stopWords.count(word);
                if(count == 0 && word != " "){
                    wordfreq[word]++;
                }
            }
        }
    }
    //5.将unordered_map<string,int>的内容存入_dict中
    for(const pair<string,int>& elem : wordfreq){
        _dict.push_back(elem);
    }
}

void DictProducer::buildCnDict(){
    /* std::cout << "buildCnDict :" <<std::endl; */
    //用来保存单词及其出现的次数
    unordered_map<string,int> wordfreq;
    for(string filename:_conf._zhFiles){
        //1.打开文件
        ifstream ifs(filename);
        if(!ifs.good()){
            std::cout << "open " << filename << " fail "<<std::endl;
            return ;
        }
        //2.从文件中读取出一行
        string line;
        while(getline(ifs,line)){
            //3.对数据进行清洗(标点-->空格,'\r'-->空格)
            for(int idx =0;idx <(int)line.size();){
                if(line[idx]=='\r'||ispunct(line[idx])){
                    line[idx] = ' ';
                }
                //删除'\n',因为中文词语不是以'\n'分隔的
                if(line[idx] == '\n'){
                    line.erase(idx);
                    break;
                }
                ++idx;
            }
            //4.对清洗完的数据进行切割,将单词和词频加入unordered_map中
            vector<string> words;
            _cuttor->cut(line,words);
            for(const string& word:words){
                //如果单词没有出现在停止中，就加入unordered_map
                int count = _stopWords.count(word);
                if(count == 0 && word!=" "){
                    wordfreq[word]++;
                }
            }
        }
    }
    //5.将unordered_map<string,int>的内容存入_dict中
    for(const pair<string,int>& elem : wordfreq){
        _dict.push_back(elem);
    }

}

void DictProducer::createIndex(){
    for(int idx =0;idx<(int)_dict.size();++idx){
        //取出单词
        string word = _dict[idx].first;
        //取出字母或者汉字
        for(int cidx=0;cidx<(int)word.size();){
            int nBytes = nBytesCode(word[cidx]);
            string character = word.substr(cidx,nBytes);
            cidx += nBytes;
        
            //建立字母与单词关系的索引库
            _index[character].insert(idx);
        }
    } 
}

void DictProducer::store(){
    //把_dict存入磁盘中
    //1.打开文件
    ofstream ofs1("../data/dict.dat");
    //2.将_dict的内容写入文件
    for(int idx =0;idx < (int)_dict.size();++idx){
        ofs1 << idx <<" " << _dict[idx].first << " " << _dict[idx].second << "\n";
    }
    //3.关闭文件
    ofs1.close();
    //把_index存入磁盘中
    //1.打开文件
    ofstream ofs2("../data/dictIndex.dat");
    //2.将_index的内容写入磁盘
    for(const pair<string,set<int>>& elem:_index){
        ofs2 << elem.first << " ";
        for(int idx:elem.second){
            ofs2<< idx << " ";
        }
        ofs2 << "\n";
    }
    //3.关闭文件
    ofs2.close();
}

void DictProducer::start(){
    LogInfo("Build configuration information");
    _conf.buildConf();
    LogInfo("Build stopWords set");
    buildStopSet();
    LogInfo("Build English dictionary");
    buildEnDict();
    LogInfo("Build Chinese dictionary");
    buildCnDict();
    LogInfo("Create index library");
    createIndex();
    LogInfo("Information is persisted to disk");
    store(); 
    Mylogger::destory();
}
